package com.software.process

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object CountWords {
  def main(args: Array[String]): Unit = {
    //1.创建Spark运行配置对象
    val sparkConf = new SparkConf().setMaster("local").setAppName("WordCountExample").set("spark.testing.memory", "2147480000")

    //2.创建Spark上下文环境对象
    var sc: SparkContext = new SparkContext(sparkConf)
    //    val inputFile = args(0)
    val inputFile = "input\\words.txt"
    //3.读取文件数据（外部源文件）
    var fileRdd: RDD[String] = sc.textFile(inputFile)
    //4.按照空格切分文件中的单词
    var wordSplitRDD: RDD[String] = fileRdd.flatMap(data => data.split(" "))
    //var wordsSplitResult =fileRdd.flatMap(data => data.split(" "))
    //wordsSplitResult.foreach(println)
    //5.转换结构(单词，初始值：1)
    var wordRddInit: RDD[(String, Int)] = wordSplitRDD.map(data => (data, 1))
    var wordTotal: RDD[(String, Int)] = wordRddInit.reduceByKey(_ + _)

    var tuples: Array[(String, Int)] = wordTotal.collect()
    //输出结果
    tuples.foreach(println)
    //关闭SparkContext对象
    sc.stop()
  }
}
